// Derived comment from V8 sources:
// TOKEN_LIST takes a list of 3 macros M, all of which satisfy the
// same signature M(name, string, precedence), where name is the
// symbolic token name, string is the corresponding syntactic symbol
// (or null, for literals), and precedence is the precedence (or 0).
// The parameters are invoked for token categories as follows:
//
//   T: Non-keyword tokens
//   K: Keyword tokens
//   F: Future (reserved) keyword tokens

j8.prototype.initTokens = function() {
  var index = 0;
  var stringToToken = {};
  var nameToToken = {};

  var createToken = function(name, string, precedence, type) {
    var result = {"name":name, "string":string, "precedence":precedence, "type":type, "index":index++};
    stringToToken[string] = result;
    nameToToken[name] = result;
    return result;
  };

  var T = function(name, string, precedence) {
    return createToken(name, string, precedence, "T");
  };

  var K = function(name, string, precedence) {
    return createToken(name, string, precedence, "K");
  };

  var F = function(name, string, precedence) {
    return createToken(name, string, precedence, "F");
  };

  this.tokenList = [
  /* End of source indicator. */
  T("EOS", "EOS", 0),

  /* Punctuators (ECMA-262, section 7.7, page 15). */
  T("LPAREN", "(", 0),
  T("RPAREN", ")", 0),
  T("LBRACK", "[", 0),
  T("RBRACK", "]", 0),
  T("LBRACE", "{", 0),
  T("RBRACE", "}", 0),
  T("COLON", ":", 0),
  T("SEMICOLON", ";", 0),
  T("PERIOD", ".", 0),
  T("CONDITIONAL", "?", 3),
  T("INC", "++", 0),
  T("DEC", "--", 0),

  /* Assignment operators. */
  /* IsAssignmentOp() relies on this block of enum values */
  /* being contiguous and sorted in the same order! */
  T("INIT_VAR", "=init_var", 2),  /* AST-use only. */
  T("INIT_CONST", "=init_const", 2),  /* AST-use only. */
  T("ASSIGN", "=", 2),
  T("ASSIGN_BIT_OR", "|=", 2),
  T("ASSIGN_BIT_XOR", "^=", 2),
  T("ASSIGN_BIT_AND", "&=", 2),
  T("ASSIGN_SHL", "<<=", 2),
  T("ASSIGN_SAR", ">>=", 2),
  T("ASSIGN_SHR", ">>>=", 2),
  T("ASSIGN_ADD", "+=", 2),
  T("ASSIGN_SUB", "-=", 2),
  T("ASSIGN_MUL", "*=", 2),
  T("ASSIGN_DIV", "/=", 2),
  T("ASSIGN_MOD", "%=", 2),

  /* Binary operators sorted by precedence. */
  /* IsBinaryOp() relies on this block of enum values */
  /* being contiguous and sorted in the same order! */
  T("COMMA", ",", 1),
  T("OR", "||", 4),
  T("AND", "&&", 5),
  T("BIT_OR", "|", 6),
  T("BIT_XOR", "^", 7),
  T("BIT_AND", "&", 8),
  T("SHL", "<<", 11),
  T("SAR", ">>", 11),
  T("SHR", ">>>", 11),
  T("ADD", "+", 12),
  T("SUB", "-", 12),
  T("MUL", "*", 13),
  T("DIV", "/", 13),
  T("MOD", "%", 13),

  /* Compare operators sorted by precedence. */
  /* IsCompareOp() relies on this block of enum values */
  /* being contiguous and sorted in the same order! */
  T("EQ", "==", 9),
  T("NE", "!=", 9),
  T("EQ_STRICT", "===", 9),
  T("NE_STRICT", "!==", 9),
  T("LT", "<", 10),
  T("GT", ">", 10),
  T("LTE", "<=", 10),
  T("GTE", ">=", 10),
  K("INSTANCEOF", "instanceof", 10),
  K("IN", "in", 10),

  /* Unary operators. */
  /* IsUnaryOp() relies on this block of enum values */
  /* being contiguous and sorted in the same order! */
  T("NOT", "!", 0),
  T("BIT_NOT", "~", 0),
  K("DELETE", "delete", 0),
  K("TYPEOF", "typeof", 0),
  K("VOID", "void", 0),

  /* Keywords (ECMA-262, section 7.5.2, page 13). */
  K("BREAK", "break", 0),
  K("CASE", "case", 0),
  K("CATCH", "catch", 0),
  K("CONTINUE", "continue", 0),
  K("DEBUGGER", "debugger", 0),
  K("DEFAULT", "default", 0),
  /* DELETE */
  K("DO", "do", 0),
  K("ELSE", "else", 0),
  K("FINALLY", "finally", 0),
  K("FOR", "for", 0),
  K("FUNCTION", "function", 0),
  K("IF", "if", 0),
  /* IN */
  /* INSTANCEOF */
  K("NEW", "new", 0),
  K("RETURN", "return", 0),
  K("SWITCH", "switch", 0),
  K("THIS", "this", 0),
  K("THROW", "throw", 0),
  K("TRY", "try", 0),
  /* TYPEOF */
  K("VAR", "var", 0),
  /* VOID */
  K("WHILE", "while", 0),
  K("WITH", "with", 0),

  /* Future reserved words (ECMA-262, section 7.5.3, page 14). */
  F("ABSTRACT", "abstract", 0),
  F("BOOLEAN", "boolean", 0),
  F("BYTE", "byte", 0),
  F("CHAR", "char", 0),
  F("CLASS", "class", 0),
  K("CONST", "const", 0),
  F("DOUBLE", "double", 0),
  F("ENUM", "enum", 0),
  F("EXPORT", "export", 0),
  F("EXTENDS", "extends", 0),
  F("FINAL", "final", 0),
  F("FLOAT", "float", 0),
  F("GOTO", "goto", 0),
  F("IMPLEMENTS", "implements", 0),
  F("IMPORT", "import", 0),
  F("INT", "int", 0),
  F("INTERFACE", "interface", 0),
  F("LONG", "long", 0),
  K("NATIVE", "native", 0),
  F("PACKAGE", "package", 0),
  F("PRIVATE", "private", 0),
  F("PROTECTED", "protected", 0),
  F("PUBLIC", "public", 0),
  F("SHORT", "short", 0),
  F("STATIC", "static", 0),
  F("SUPER", "super", 0),
  F("SYNCHRONIZED", "synchronized", 0),
  F("THROWS", "throws", 0),
  F("TRANSIENT", "transient", 0),
  F("VOLATILE", "volatile", 0),

  /* Literals (ECMA-262, section 7.8, page 16). */
  K("NULL_LITERAL", "null", 0),
  K("TRUE_LITERAL", "true", 0),
  K("FALSE_LITERAL", "false", 0),
  T("NUMBER", null, 0),
  T("STRING", null, 0),

  /* Identifiers (not keywords or future reserved words). */
  T("IDENTIFIER", null, 0),

  /* Illegal token - not able to scan. */
  T("ILLEGAL", "ILLEGAL", 0),

  /* Scanner-internal use only. */
  T("COMMENT", null, 0),

  ];

  var $T = this.$T = this.getTokenByName = function(name) {
    return nameToToken[name];
  };
  
  var $TI = this.$TI = function(name) {
    return $T(name).index;
  };

  // Returns the token with specified string if str is a keyword;
  // returns IDENTIFIER otherwise. 
  this.getTokenByString = function(str) {
    var result = stringToToken[str];
    if (typeof(result) === "undefined") {
      return $T("IDENTIFIER");
    }
    return result;
  };

  var isTokenInRange = function(token, nameFrom, nameTo) {
    return $TI(nameFrom) <= token.index && token.index <= $TI(nameTo);
  };

  this.isAssignmentOp = function(token) {
    return isTokenInRange(token, "INIT_VAR", "ASSIGN_MOD");
  };

  this.isBinaryOp = function(token) {
    return isTokenInRange(token, "COMMA", "MOD");
  };

  this.isCompareOp = function(token) {
    return isTokenInRange(token, "EQ", "IN");
  };

  this.isBitOp = function(token) {
    return isTokenInRange(token, "BIT_OR", "SHR") || token.index == $TI("BIT_NOT");
  };

  this.isUnaryOp = function(token) {
    return isTokenInRange(token, "NOT", "VOID") || token.index == $TI("ADD") || token.index == $TI("SUB");
  };

  this.isCountOp = function(token) {
    return token.index == $TI("INC") || $TI("DEC");
  };
};